import scrapy
from pandas import json
from scrapy.http import Request
from bs4 import BeautifulSoup
from lianjia.items import LianjiaItem
from selenium.webdriver.firefox.options import Options
from selenium import webdriver


class Lianjia(scrapy.spiders.Spider):
    name = "lj"
    allowed_domains = ['sh.lianjia.com']
    baseUrl = 'https://sh.lianjia.com/zufang/'  #

    def start_requests(self):
        yield Request(self.baseUrl, self.parse)

    def parse(self, response):
        options = Options()
        options.add_argument('-headless')  # 无头参数
        driver = webdriver.Chrome()
        driver.get(self.baseUrl)
        driver.maximize_window()
        soup = BeautifulSoup(driver.page_source, 'lxml').find('div', {'class': 'content__list'}).find_all('div')
        for home in soup:
            item = LianjiaItem()
            item['pic'] = str(home.find('a', {'class': 'content__list--item--aside'}).find('img')['src']).strip()
            item['name'] = str(home.find('div', {'class': 'content__list--item--main'}).find('p').get_text()).strip()
            # item['sizeDes'] = str(home.find('span', {'class': 'zone'}).find('span').get_text()).strip()
            # item['meters'] = str(home.find('span', {'class': 'meters'}).get_text()).strip()
            print(json.dumps(item))
            yield item

